/** 
 * 项目名称:Sprider-WebMagic-bak 
 * 文件名称:MaoYanSearchPageProcessor.java 
 * 包名:com.xiaotu.spider.maoyan.pageprocessor 
 * 创建日期:2020年1月2日上午9:28:16 
 * Copyright (c) 2020, zxaiye1314@163.com All Rights Reserved.</pre> 
 */  
package com.xiaotu.spider.maoyan.pageprocessor;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.apache.log4j.spi.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.model.MaoYanMovieBasicInfoModel;
import com.xiaotu.common.model.MaoYanMoviePeopleModel;
import com.xiaotu.common.model.MaoYanMovieScoreRelateModel;
import com.xiaotu.common.model.MaoYanMovieSplitModel;
import com.xiaotu.common.model.MaoYanSearchMovieModel;
import com.xiaotu.common.util.CacheHandler;
import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.DataRedisKeyUtils;
import com.xiaotu.common.util.DateUtils;
import com.xiaotu.common.util.GsonUtils;
import com.xiaotu.common.util.HtmlUtil;
import com.xiaotu.common.util.KeywordEncode;
import com.xiaotu.common.util.PageUtils;
import com.xiaotu.common.util.PropertiesUtil;
import com.xiaotu.common.util.RegexUtils;
import com.xiaotu.common.util.SepratorUtil;
import com.xiaotu.common.util.StringUtil;
import com.xiaotu.common.util.UUIDUtils;
import com.xiaotu.spider.APageProcessor;
import net.sf.json.JSONObject;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.selector.Html;

/** 
 * 项目名称：Sprider-WebMagic    
 * 类名称：MaoYanSearchPageProcessor    
 * 类描述：    
 * 创建人：张晓   
 * 创建时间：2020年1月2日 上午9:28:16    
 * @version   
 */
@Component("MaoYanSearchMoviePageProcessor")
public class MaoYanSearchMoviePageProcessor extends APageProcessor {
	
	@Autowired
	private CacheHandler cacheHandler;
	
	private Map<String,String> cookies = new HashMap<String,String>();
	
	{ 
		cookies.put("Cookie", PropertiesUtil.getProperty("BoxOffice.site.Cookie"));
	}
	
	public static final String BASE_SEARCH_URL = "https://piaofang.maoyan.com/search?key="; 
	
	public static final String PEOPLE_URL = "https://piaofang.maoyan.com/movie/%s/wantindex?wkwebview=1&city_tier=0&city_id=0&cityName=全国";//受众票房
	
	public static final String BASEINFO_URL = "http://piaofang.maoyan.com/movie/%s?_v_=yes";//基础信息
	
	
	@SuppressWarnings("unchecked")
	@Override
	protected Request getStartRequest(JSONObject target) {
		Request request = new Request(BASE_SEARCH_URL
				+ KeywordEncode.encode(target.getString("tvName")));
		request.setExtras((Map<String, Object>)target);
		return request;
	}
	
	@Override
	public void process(Page page) {
		
		Map<String, Object> extrasMap = page.getRequest().getExtras();
		
		switch (this.getPageDataType(page))
        {
            case Constants.PageDataType.SEARCH_LIST:
                this.matchTvProcess(page,extrasMap);
                break;
            case Constants.PageDataType.BASICINFO:
                this.basicinfoProcess(page,extrasMap);
                break;
            case Constants.PageDataType.TV_AUDIENCE:
                this.peopleProcess(page,extrasMap);
                break;	
        }
	}

	@Override
	public String getPageDataType(Page page) {
		String url = page.getUrl().toString();
        if (url.contains("_v_=yes")){
        	return Constants.PageDataType.BASICINFO;
        }else if(url.contains("/wantindex")){
			return Constants.PageDataType.TV_AUDIENCE;
		}else if(url.contains("search?key")){
			return Constants.PageDataType.SEARCH_LIST;
		}
        return null;
	}
	
	@Override
	protected String getSpiderType() {
		return Constants.SpiderTaskType.MOVIE_BOXOFFICE;
	}
	
	/**
	 * matchTvProcess(与猫眼电影进行匹配)   
	 * 创建人：张晓   
	 * 创建时间：2020年1月2日 下午1:55:44    
	 * 修改备注： 
	 * @param page
	 * @param extrasMap
	 */
	@SuppressWarnings("unused")
	private void matchTvProcess(Page page, Map<String, Object> extraMap) {
		
		//目标剧名称
		String target_tvName = extraMap.get("tvName")==null?null:extraMap.get("tvName").toString();
		//目标剧导演
		String target_director = extraMap.get("director")==null?null:extraMap.get("director").toString().replaceAll("[,，/]", ",");
		//目标剧主演
		String target_actor = extraMap.get("actor")==null?null:extraMap.get("actor").toString().replaceAll("[,，/]", ",");
		
		List<String> articleTagList = page.getHtml().xpath("//div[@id='search-list']//article").all();
		
		List<Map<String,Object>> appearanceMapList = new ArrayList<Map<String,Object>>();
		
		//正确率最高的
		String maxAppearanceMaoDataId = "";
		String maxAppearance = "";
		String maxAppearanceTvname = "";
		
		for(String article : articleTagList){
			
			Map<String,Object> appearanceMap = new HashMap<String,Object>();
			
			Html articleHtml = new Html(article);
			//猫眼剧名称
			String mao_tvName = null;
			//猫眼剧上映时间
			String mao_tvRelaseDate = null;
			//猫眼剧导演
			String mao_director = null;
			//猫眼剧主演
			String mao_actor = null;
			//猫眼剧的网站数据id
			String mao_dataId = null;
			
			//猫眼剧名称
			mao_tvName = articleHtml.xpath("//div[@class='title']//text()").get().trim();
			//猫眼剧上映时间
			if(null != articleHtml.xpath("//p//text()").all().get(0)){
				mao_tvRelaseDate = articleHtml.xpath("//p//text()").all().get(0).trim().replaceAll("[\u4e00-\u9fa5- ]", "");
			}
			
			//猫眼剧导演
			if(null != articleHtml.xpath("//p[@class='lineDot']//span[@class='movie-director']//text()").get()){
				mao_director = convertStr(articleHtml.xpath("//p[@class='lineDot']//span[@class='movie-director']//text()").get().trim());
			}
			
			//猫眼剧主演
			if(null == mao_director && null != articleHtml.xpath("//p[@class='lineDot']//span//text()").all().get(0)){
				mao_actor = convertStr(articleHtml.xpath("//p[@class='lineDot']//span//text()").all().get(0).trim());
			}
			if(null != mao_director && null != articleHtml.xpath("//p[@class='lineDot']//span//text()").all().get(1)){
				mao_actor = convertStr(articleHtml.xpath("//p[@class='lineDot']//span//text()").all().get(1).trim());
			}
			
			//猫眼剧的网站数据id
			if(null != articleHtml.xpath("//article[@class='indentInner canTouch']/@data-url").get()){
				mao_dataId = articleHtml.xpath("//article[@class='indentInner canTouch']/@data-url").get().replaceAll("[a-z/]", "");
			}
			
			
			int appearance = 0;
			//剧名
			if(StringUtils.isNotBlank(mao_tvName) && StringUtils.isNotBlank(target_tvName)){
				
				mao_tvName = StringUtil.matchesStr(mao_tvName, "[\u4e00-\u9fa5\\s]+")==false?StringUtil.stringFilter(mao_tvName):mao_tvName;
				
				target_tvName = StringUtil.matchesStr(target_tvName, "[\u4e00-\u9fa5\\s]+")==false?StringUtil.stringFilter(target_tvName):target_tvName;
				
				if(mao_tvName.equals(target_tvName)){
					
					appearance++;
					
					//导演
					if(StringUtils.isNotBlank(target_director) && StringUtils.isNotBlank(mao_director)
							&& (isContains(mao_director,target_director) || isContains(target_director,mao_director))){
						appearance++;
					}
					//演员
					if(StringUtils.isNotBlank(target_actor) && StringUtils.isNotBlank(mao_actor)
							&& (isContains(mao_actor,target_actor) || isContains(target_actor,mao_actor))){
						appearance++;
					}
					
				}
			}
			
			appearanceMap.put("mao_dataId", mao_dataId);
			appearanceMap.put("appearance", appearance);
			appearanceMap.put("mao_tvName", mao_tvName);
			
			appearanceMapList.add(appearanceMap);
		}
		
		if(appearanceMapList.size()>0){
			Collections.sort(appearanceMapList, new Comparator<Map<String,Object>>() {

				@Override
				public int compare(Map<String, Object> o1, Map<String, Object> o2) {
					
					int v1 = Integer.parseInt(o1.get("appearance").toString()); 
					int v2 = Integer.parseInt(o2.get("appearance").toString());
					
						if((v2-v1)>0){
							return 1;
						}else if((v2-v1)<0){
							return -1;
						}else{
							return 0;
						}
				}
			});
			
			//正确率最高的
			maxAppearanceMaoDataId = appearanceMapList.get(0).get("mao_dataId").toString();
			maxAppearance = appearanceMapList.get(0).get("appearance").toString();
			maxAppearanceTvname = appearanceMapList.get(0).get("mao_tvName").toString();
			
			int appearance = Integer.parseInt(appearanceMapList.get(0).get("appearance").toString());
			if(appearance>=1){//最低条件名字匹配
				maxAppearanceMaoDataId = appearanceMapList.get(0).get("mao_dataId").toString();
				if(StringUtils.isNotBlank(maxAppearanceMaoDataId)){
					//爬取猫眼其他相关信息
					extraMap.put("dataId", maxAppearanceMaoDataId);
					//爬取猫眼
					PageUtils.addTargetRequest(page, String.format(BASEINFO_URL, maxAppearanceMaoDataId), extraMap);
				}
			}
		}else{
			//将未匹配到的进行记录
			String key = "NoMatchMovie_"+DateUtils.fomartDateToString(new Date(), DateUtils.FILE_NOMENCLATURE_FORMAT);
        	cacheHandler.hSet(key, UUIDUtils.getId()+"_"+target_tvName, extraMap);
        	cacheHandler.setCacheTimeout(key);
		}
		
	}
	
	
	/***
	 * basicinfoProcess(基本信息)   
	 * 创建人：张晓   
	 * 创建时间：2020年1月2日 下午1:55:24    
	 * 修改备注： 
	 * @param page
	 * @param extrasMap
	 */
	private void basicinfoProcess(Page page, Map<String, Object> extraMap) {
		
		String url = page.getRequest().getUrl();
		
		String dataId = extraMap.get("dataId").toString();
		
		String tvName = extraMap.get("tvName").toString();
		
		if(url.contains(dataId)){
			//解密数据
	        Html html = PageUtils.spiderAudienceRating(page,cookies);
	        
	        //票房相关
	        String totalBoxoffice = null;// 累计票房
	        String firstDayBoxoffice = null;// 首日票房
	        String firstWeekBoxoffice = null;// 首周票房
	        String forecastBoxoffice = null;// 预测票房
	        String boxoffice = null; // 分账总票房
	        String productionBox = null;//片方票房
	        String cinemaBox = null;//影院票房
	        String otherBox = null;//其他票房
	        
	        List<String> checkDataList = new ArrayList<String>();
	        
			//英文名
			String englishName = null;
			//类型
			String [] label = null;
			//地区
			String [] area = null;
			//时长
			String timeLength = null;
			//大陆上映地区
			String showingArea = null;
			String[] dateAndArea = new String[2];
			
			try {
				
				//英文名
				englishName = page.getHtml().xpath("//div[@class='info-detail']//p[@class='info-etitle']//span[@class='info-etitle-content']//text()").get();
				
				String labelStr = page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='detail-list']"
						+ "/div[@class='detail-list-content']//p[@class='info-category']//text()").get();
				//补充类型
				if(null == labelStr){
					if(null != page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='detail-list']//p").all()
							&& page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='detail-list']//p").all().size()>0){
						labelStr = HtmlUtil.delHTMLTag(page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='detail-list']//p").all().get(0));
					}
				}
				
				if(StringUtils.isNotBlank(labelStr)){
					label = labelStr.replaceAll("[/]", ",").replaceAll("[ ]", "").split(",");
					if(label.length == 0){
						label = null;
					}
				}
				
				String arealStr = page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='detail-list']"
						+ "/div[@class='detail-list-content']//div[@class='info-source-duration']//p[@class='.ellipsis-1']//text()").get();
				if(StringUtils.isNotBlank(arealStr)){
					area = arealStr.replace("/", "").replace("\"", "").trim().split(",");
					if(area.length == 0){
						area = null;
					}
				}
				//时长
				timeLength = page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='detail-list']"
						+ "/div[@class='detail-list-content']//div[@class='info-source-duration']//span//text()").get();
				
				//补充时长
				if(null == timeLength){
					if(null!= page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='detail-list']//p").all()
							&& page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='detail-list']//p").all().size()==2){
						timeLength = HtmlUtil.delHTMLTag(page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='detail-list']//p").all().get(1));
					}
				}
				
				if(StringUtils.isNotBlank(timeLength) && timeLength.indexOf("集数")==-1){
					timeLength = timeLength.replaceAll("[\u4e00-\u9fa5]", "").trim();
				}else{
					timeLength = null;
				}
				
				//大陆上映地区
				showingArea = page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='detail-list']"
						+ "/div[@class='detail-list-content']//div[@class='info-release']//span[@class='score-info ellipsis-1']//text()").get();
				
				if(null == showingArea){
					showingArea = page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='detail-list']"
							+ "/div[@class='detail-list-content']//div[@class='info-source-duration']//p//text()").get();
				}
				
				String showingDate = null;
				if(null == showingArea){
					if(null != page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='info-list']//p").all()
							&& page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='info-list']//p").all().size()==3){
						showingDate = HtmlUtil.delHTMLTag(page.getHtml().xpath("//div[@class='info-detail']//div[@class='info-detail-extra']/div[@class='info-list']//p").all().get(2));
						if(showingDate.lastIndexOf("上线")!=-1){
							showingDate = showingDate.substring(0,showingDate.indexOf("上线"));
						}
						showingDate = showingDate.replaceAll("[\u4e00-\u9fa5-]", "");
						if(showingDate.length()==8){
							showingDate = convertDateStr(showingDate);
							dateAndArea[0] = showingDate;
						}
					}
				}
				if(null != showingArea && StringUtils.isNotBlank(showingArea)){
					dateAndArea = matchShowingAreaAndShowingDate(HtmlUtil.delHTMLTag(showingArea));
				}
				
	            
	            //累计票房相关
	            checkDataList = html.xpath
	            		("//div[@class='movie-container']//div[@class='info-detail-row']").all();
	            if (checkDataList.size()>0){
	            	String boxOfficeLine =html.xpath
		            		("//div[@class='movie-container']//div[@class='info-detail-row']").all().get(0);
		            Html htmlUp = new Html(boxOfficeLine);
		            List<String> allUp = htmlUp.xpath("//div[@class='info-detail-row']//div").all();
		            if(new Html(allUp.get(0)).xpath("//p[@class='info-detail-title']").get().trim().contains("累计票房")){
		            	totalBoxoffice = getFormattedData(new Html(allUp.get(0)).xpath("//p[@class='info-detail-content']").get()).replace("--", "").replace("暂无", "");
		            }
		            if(new Html(allUp.get(2)).xpath("//p[@class='info-detail-title']").get().trim().contains("首日票房")){
		            	firstDayBoxoffice = getFormattedData(new Html(allUp.get(2)).xpath("//p[@class='info-detail-content']").get()).replace("--", "").replace("暂无", "");
		            }
		            if(new Html(allUp.get(4)).xpath("//p[@class='info-detail-title']").get().trim().contains("首周票房")){
		            	firstWeekBoxoffice = getFormattedData(new Html(allUp.get(4)).xpath("//p[@class='info-detail-content']").get()).replace("--", "").replace("暂无", "");
		            }
		            if(new Html(allUp.get(6)).xpath("//p[@class='info-detail-title']").get().trim().contains("预测票房")){
		            	forecastBoxoffice = getFormattedData(new Html(allUp.get(6)).xpath("//p[@class='info-detail-content']").get()).replace("--", "").replace("暂无", "");
		            }
		            
		            
		            //分账票房相关
		            String boxOffice =html.xpath
		            		("//div[@class='movie-container']//div[@class='info-detail-row']").all().get(1);
		            Html htmls = new Html(boxOffice);
		            List<String> all = htmls.xpath("//div[@class='info-detail-row']//div").all();
		            if (all != null && all.size() == 8){
		            	if(new Html(all.get(0)).xpath("//p[@class='info-detail-title']").get().trim().contains("分账票房")){
			            	boxoffice = getFormattedData(new Html(all.get(0)).xpath("//p[@class='info-detail-content']").get()).replace("--", "").replace("暂无", ""); 
			            }
			            if(new Html(all.get(3)).xpath("//p[@class='info-detail-title']").get().trim().contains("片方")){
			            	productionBox = getFormattedData(new Html(all.get(3)).xpath("//p[@class='info-detail-content']").get()).replace("--", "").replace("暂无", "");
			            }
			            if(new Html(all.get(5)).xpath("//p[@class='info-detail-title']").get().trim().contains("影院")){
			            	cinemaBox = getFormattedData(new Html(all.get(5)).xpath("//p[@class='info-detail-content']").get()).replace("--", "").replace("暂无", "");
			            }
			            if(new Html(all.get(7)).xpath("//p[@class='info-detail-title']").get().trim().contains("其他")){
			            	otherBox = getFormattedData(new Html(all.get(7)).xpath("//p[@class='info-detail-content']").get()).replace("--", "").replace("暂无", "");
			            }
		            }
	            }
	            
			} catch (Exception e) {
				System.out.println("ERROR   basicinfoProcess()当前异常信息...........当前url链接为："+url);
			}
			
			MaoYanSearchMovieModel maoYanSearchMovieModel = new MaoYanSearchMovieModel(
					UUIDUtils.getId(), null, dataId, tvName, englishName, label, area, timeLength,
					dateAndArea[1], dateAndArea[0], totalBoxoffice, firstDayBoxoffice,
					firstWeekBoxoffice, forecastBoxoffice, boxoffice, productionBox,
					cinemaBox, otherBox, null, null, null,
					null, null, null, null, null,
					null, null, null, null
					);
			extraMap.put("maoYanSearchMovieModel", maoYanSearchMovieModel);
			PageUtils.addTargetRequest(page, String.format(PEOPLE_URL, dataId), extraMap);
		}
		
	}
	

	/**
	 * peopleProcess(人群受众)   
	 * 创建人：张晓   
	 * 创建时间：2020年1月2日 下午1:55:07    
	 * 修改备注： 
	 * @param page
	 * @param extrasMap
	 */
	private void peopleProcess(Page page,Map<String, Object> extraMap)
    {
		String url = page.getRequest().getUrl();
		
		String dataId = extraMap.get("dataId").toString();
		
		MaoYanSearchMovieModel maoYanSearchMovieModel = (MaoYanSearchMovieModel)extraMap.get("maoYanSearchMovieModel");
		
		List<String> ageNameList = new ArrayList<>();
		List<String> areaNameList = new ArrayList<>();
		List<String> ageValueList = new ArrayList<>();
		List<String> areaValueList = new ArrayList<>();
		
		String[][] sexData = new String[2][];
		String[][] eduData = new String[2][];
		String[][] occpData = new String[2][];
		String[][] labelData = new String[2][];
		if(url.contains(dataId)){
			
			try {
				
				Html html = PageUtils.spiderAudienceRating(page,cookies);
		        
		        List<String> divList = html.xpath("//div[@class='stackcolumn']").all();
		        
		        if (divList != null && !divList.isEmpty()){
		        	ageNameList = html.xpath("//svg[@id='ageData']/g[@class='x']//text/text()")
			                .all();
			        ageNameList = clearSpaces(ageNameList);
			        ageValueList = html.xpath
			                ("//svg[@id='ageData']/g[@class='bar-group']//text/text()").all();
			        ageValueList = clearSpaces(ageValueList);
			        //this.namesAndValuesValidate(ageNameList, ageValueList);
			        //转换数据
			        Map<String, List<String>> convertNamesAndValuesMap = convertNamesAndValues(ageNameList,ageValueList);
			        ageNameList = convertNamesAndValuesMap.get("nameList");
			        ageValueList = convertNamesAndValuesMap.get("valueList");
			        
			        
			        sexData = this.getLineNamesAndValues(divList, "性别");
			        eduData = this.getLineNamesAndValues(divList, "教育");

			        areaNameList = html.xpath
			                ("//section[@class='area-section']//div[@class='linebars-content']/div[2]/text()").all();
			        areaNameList = clearSpaces(areaNameList);
			        areaValueList = html.xpath
			                ("//section[@class='area-section']//div[@class='linebars-content']//div[@class='linebars-value']/text()").all();
			        areaValueList = clearSpaces(areaValueList);
			        //this.namesAndValuesValidate(areaNameList, areaValueList);
			        //转换数据
			        Map<String, List<String>> convertMap = convertNamesAndValues(areaNameList,areaValueList);
			        areaNameList = convertMap.get("nameList");
			        areaValueList = convertMap.get("valueList");
			        
			        occpData = this.getPieNamesAndValues(html, "occupData");
			        labelData = this.getPieNamesAndValues(html, "interestData");
		        }
				
			} catch (Exception e) {
				System.out.println("ERROR   peopleProcess()当前异常信息...........当前url链接为："+url);
			}

	        maoYanSearchMovieModel.setAgeNames(ageNameList.toArray(new String[ageNameList.size()]));
	        maoYanSearchMovieModel.setAgeRates(ageValueList.toArray(new String[ageValueList.size()]));
	        maoYanSearchMovieModel.setAreaNames(areaNameList.toArray(new String[areaNameList.size()]));
	        maoYanSearchMovieModel.setAreaRates(areaValueList.toArray(new String[areaValueList.size()]));
	        maoYanSearchMovieModel.setEduNames(eduData[0]);
	        maoYanSearchMovieModel.setEduRates(eduData[1]);
	        maoYanSearchMovieModel.setSexNames(sexData[0]);
	        maoYanSearchMovieModel.setSexRates(sexData[1]);
	        maoYanSearchMovieModel.setOccpName(occpData[0]);
	        maoYanSearchMovieModel.setOccpRates(occpData[1]);
	        maoYanSearchMovieModel.setLabelNames(labelData[0]);
	        maoYanSearchMovieModel.setLabelRates(labelData[1]);
	        page.putField(MaoYanSearchMovieModel.class.getName(), maoYanSearchMovieModel);
		}
		
    }
	
	
	private List<String> clearSpaces(List<String> paramlist){
    	List<String> newList = new ArrayList<String>(paramlist.size());
    	for(String str : paramlist){
    		newList.add(str.trim());
    	}
    	return newList;
    }
	
	private void namesAndValuesValidate(List<String> nameList, List<String> valueList)
    {
        if (nameList == null || nameList.isEmpty())
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        if (valueList == null || valueList.isEmpty() || nameList.size() != valueList.size())
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
    }
	
	private Map<String,List<String>> convertNamesAndValues(List<String> nameList,List<String> valueList){
    	Map<String,List<String>> dataMap = new HashMap<String,List<String>>();
    	String[] returnNameArr = new String[nameList.size()];
    	String[] returnValueArr = new String[valueList.size()];
    	for(int i=0;i<nameList.size();i++){
    		if(nameList.get(i).contains("二线城市")){
    			returnNameArr[0] = nameList.get(i);
    			returnValueArr[0] = valueList.get(i);
        	}
        	else if(nameList.get(i).contains("四线城市")){
        		returnNameArr[1] = nameList.get(i);
    			returnValueArr[1] = valueList.get(i);
        	}
        	else if(nameList.get(i).contains("三线城市")){
        		returnNameArr[2] = nameList.get(i);
    			returnValueArr[2] = valueList.get(i);
        	}
        	else if(nameList.get(i).contains("一线城市")){
        		returnNameArr[3] = nameList.get(i);
    			returnValueArr[3] = valueList.get(i);
        	}
        	else if(nameList.get(i).contains("白领")){
        		returnNameArr[0] = nameList.get(i);
    			returnValueArr[0] = valueList.get(i);
        	}
        	else if(nameList.get(i).contains("其他")){
        		returnNameArr[1] = nameList.get(i);
    			returnValueArr[1] = valueList.get(i);
        	}
        	else if(nameList.get(i).contains("学生")){
        		returnNameArr[2] = nameList.get(i);
    			returnValueArr[2] = valueList.get(i);
        	}
        	else if(nameList.get(i).contains("20岁以下")){
        		returnNameArr[0] = nameList.get(i);
    			returnValueArr[0] = valueList.get(i);
        	}else if(nameList.get(i).contains("20~24")){
        		returnNameArr[1] = nameList.get(i);
    			returnValueArr[1] = valueList.get(i);
        	}else if(nameList.get(i).contains("25~29")){
        		returnNameArr[2] = nameList.get(i);
    			returnValueArr[2] = valueList.get(i);
        	}else if(nameList.get(i).contains("30~34")){
        		returnNameArr[3] = nameList.get(i);
    			returnValueArr[3] = valueList.get(i);
        	}else if(nameList.get(i).contains("35~39")){
        		returnNameArr[4] = nameList.get(i);
    			returnValueArr[4] = valueList.get(i);
        	}else if(nameList.get(i).contains("40岁以上")){
        		returnNameArr[5] = nameList.get(i);
    			returnValueArr[5] = valueList.get(i);
        	}
    	}
    	
    	dataMap.put("nameList", Arrays.asList(returnNameArr));
    	dataMap.put("valueList", Arrays.asList(returnValueArr));
    	return dataMap;
    	
    }
	
	private String getFormattedData(String data){
        return RegexUtils.replaceStr(HtmlUtil.delHTMLTag(data), SepratorUtil.SEP_SPACE_EN,
                StringUtils.EMPTY);
    }
	
	@SuppressWarnings({ "unused"})
	private String getStarLevel(List<String> list){
		Double starLevel = 0.0;
		if(list == null || list.size()<0)
		return null;
		for(String str: list){
			if(str.contains("fullstar")){
				starLevel +=1; 
			}else if(str.contains("halfstar")){
				starLevel+=0.5;
			}
		}
		return starLevel+"";
	}
	
	private String[] matchShowingAreaAndShowingDate(String str){
			
			String[] data = new String[2];
			if(StringUtils.isNotBlank(str)){
				if(regexStr("[\u4e00-\u9fa5-:]",str).length()>=8){
					data[0] = regexStr("[\u4e00-\u9fa5]",str);
					data[1] = regexStr("[0-9:-]",str);
				}else if (StringUtils.isEmpty(regexStr("[\u4e00-\u9fa5]",str))){
					data[1] = str.trim();
					data[0] = null;
				}else if (StringUtils.isEmpty(regexStr("[0-9:-]",str))){
					data[0] = str.trim();
					data[1] = null;
				}
			}
			
			return data;
	}
	
	private String regexStr(String regex,String str){
		
		String reg = regex;

		Pattern pat = Pattern.compile(reg);  

		Matcher mat=pat.matcher(str); 

		String returnStr = mat.replaceAll("");
		
		return returnStr;
	}
	
	private String convertDateStr(String str){
    	String first = str.substring(0,4);
    	String second = str.substring(4, 6);
    	String three = str.substring(6);
    	return first+"-"+second+"-"+three;
    }
	
	private String[][] getLineNamesAndValues(List<String> divList, String type)
    {
        String sexDiv = divList.stream().filter(divStr -> divStr.contains(type)).findFirst().get();
        if (StringUtils.isEmpty(sexDiv))
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        Html sexHtml = new Html(sexDiv);
        List<String> nameList = sexHtml.xpath("//span[@class='stackcolumn-desc-text']/text()")
                .all();
        List<String> valueList = sexHtml.xpath("//i[@class='cs']/text()").all();
        //this.namesAndValuesValidate(nameList, valueList);
        if(type.contains("性别")||type.contains("教育")){
        	String[] returnNameArr = new String[nameList.size()];
        	String[] returnValueArr = new String[valueList.size()];
        	for(int i = 0;i<nameList.size();i++){
            	
            	if(nameList.get(i).contains("男")){
            		returnNameArr[0] = nameList.get(i);
        			returnValueArr[0] = valueList.get(i);
            	}
            	else if(nameList.get(i).contains("女")){
            		returnNameArr[1] = nameList.get(i);
        			returnValueArr[1] = valueList.get(i);
            	}
            	else if(nameList.get(i).contains("本科及以上")){
            		returnNameArr[0] = nameList.get(i);
        			returnValueArr[0] = valueList.get(i);
            	}
            	else if(nameList.get(i).contains("本科以下")){
            		returnNameArr[1] = nameList.get(i);
        			returnValueArr[1] = valueList.get(i);
            	}
            }
        	nameList = Arrays.asList(returnNameArr);
        	valueList = Arrays.asList(returnValueArr);
        }
        valueList = valueList.stream().map(value -> value + "%").collect(Collectors.toList());
        return new String[][]{nameList.toArray(new String[nameList.size()]), valueList.toArray(new
                String[valueList.size()])};
    }
	
	private String[][] getPieNamesAndValues(Html html, String tagId)
    {
        List<String> occpList = html.xpath("//svg[@id='" + tagId +
                "']/path[@class='pie-item']/@data-info").all();
        if (occpList == null || occpList.isEmpty())
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        List<String> occpNameList = occpList.stream().filter(data -> data.split(";").length > 1)
                .map(data -> data.split(";")[1]).collect(Collectors.toList());
        List<String> occpValueList = occpList.stream().filter(data -> data.split(";").length > 1)
                .map(data -> data.split(";")[0]).collect(Collectors.toList());
        //this.namesAndValuesValidate(occpNameList, occpValueList);
        if(tagId.equals("occupData")){
        	//转换数据
            Map<String, List<String>> convertNamesAndValuesMap = convertNamesAndValues(occpNameList,occpValueList);
            occpNameList = convertNamesAndValuesMap.get("nameList");
            occpValueList = convertNamesAndValuesMap.get("valueList");
        }
        return new String[][]{occpNameList.toArray(new String[occpNameList.size()]),
                occpValueList.toArray(new String[occpValueList.size()])};
    }
	
	
	
	private boolean isContains(String str1,String str2){
		
		str1 = str1.replaceAll("[/,]", ",");
		
		str2 = str2.replaceAll("[/,]", ",");
		
		boolean flag = false;
		String[] str1Arr = str1.split(",");
		String[] str2Arr = str2.split(",");
		for(String s1 : str1Arr){
			for(String s2 : str2Arr){
				if(s1.contains(s2)){
					flag = true;
					return flag;
				}
			}
		}
		
		return flag;
	}
	
	private String convertStr(String paramStr){
		
		if(paramStr.length()<=0){
			return null;
		}
		if(StringUtils.isNotBlank(paramStr)){
			return paramStr.replaceAll("[导演 演员：]", "").replaceAll("[，,/]", ",");
		}
		return null;
	}
	
}
